import requests
import pandas as pd
import time
from typing import List, Dict, Any, Optional
import json
import os
from tqdm import tqdm  # 进度条库，需要安装: pip install tqdm

class EnhancedApiCaller:
    def __init__(self):
        self.base_url = "https://eco.kylinos.cn/home/compatible/index.html"
        self.session = requests.Session()
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
            'Accept': 'application/json, text/plain, */*',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
        }
        self.failed_pages = []
        
    def call_api(self, page: int, max_retries: int = 5) -> Optional[Dict[str, Any]]:
        """
        调用API接口，支持重试机制
        """
        params = {
            'page': page,
            'limit': 100,
            'query_key': '',
            'class_id_1': 3,
            'class_id_2': 21,
            'class_id_3': '130,131,166,167',
            'system_class': 1,
            'system_type[0]':2,
            'install_url[0]': 1,
            'brand[1]': '兆芯'
        }
        
        for attempt in range(max_retries):
            try:
                print(f"第 {page} 页 - 尝试 {attempt + 1}/{max_retries}")
                response = self.session.get(
                    self.base_url, 
                    params=params, 
                    headers=self.headers,
                    timeout=30
                )
                
                if response.status_code == 200:
                    data = response.json()
                    print(f"✓ 第 {page} 页调用成功")
                    return data
                else:
                    print(f"✗ 第 {page} 页HTTP错误: {response.status_code}")
                    
            except requests.exceptions.RequestException as e:
                print(f"✗ 第 {page} 页网络错误: {e}")
            except json.JSONDecodeError as e:
                print(f"✗ 第 {page} 页JSON解析错误: {e}")
            
            # 重试前等待
            if attempt < max_retries - 1:
                time.sleep(5)
        
        # 所有重试都失败
        self.failed_pages.append(page)
        return None
    
    def extract_data(self, api_data: Dict[str, Any]) -> List[Dict[str, Any]]:
        """
        从API返回数据中提取data字段
        """
        if not api_data:
            return []
        
        # 检查不同的数据结构可能性
        data_keys = ['data', 'list', 'items', 'result']
        for key in data_keys:
            if key in api_data:
                data_content = api_data[key]
                if isinstance(data_content, list):
                    return data_content
                elif isinstance(data_content, dict) and 'list' in data_content:
                    sub_data = data_content['list']
                    if isinstance(sub_data, list):
                        return sub_data
        
        print(f"警告: 未找到有效的数据列表，可用字段: {list(api_data.keys())}")
        return []
    
    def save_to_excel(self, all_data: List[Dict[str, Any]], filename: str = "api_data.xlsx"):
        """
        将数据保存到Excel文件，支持大数据量分sheet保存
        """
        if not all_data:
            print("没有数据可保存")
            return False
        
        try:
            # 如果数据量很大，可以分多个sheet保存
            max_rows_per_sheet = 10000
            if len(all_data) <= max_rows_per_sheet:
                # 数据量不大，直接保存
                df = pd.DataFrame(all_data)
                df.to_excel(filename, index=False, engine='openpyxl')
            else:
                # 数据量大，分多个sheet保存
                with pd.ExcelWriter(filename, engine='openpyxl') as writer:
                    for i in range(0, len(all_data), max_rows_per_sheet):
                        chunk = all_data[i:i + max_rows_per_sheet]
                        df = pd.DataFrame(chunk)
                        sheet_name = f"Data_{i//max_rows_per_sheet + 1}"
                        df.to_excel(writer, sheet_name=sheet_name, index=False)
            
            print(f"✓ 数据已成功保存到 {filename}")
            print(f"✓ 文件大小: {os.path.getsize(filename) / 1024 / 1024:.2f} MB")
            return True
            
        except Exception as e:
            print(f"✗ 保存Excel文件失败: {e}")
            return False
    
    def run_enhanced(self, total_pages: int = 200, start_page: int = 1):
        """
        增强版执行函数，带进度条和详细统计
        """
        print(f"开始执行API批量调用...")
        print(f"总页数: {total_pages}, 起始页: {start_page}")
        print("=" * 50)
        
        all_data = []
        successful_pages = 0
        
        # 使用进度条
        for page in tqdm(range(start_page, start_page + total_pages), desc="调用进度"):
            api_data = self.call_api(page)
            
            if api_data:
                page_data = self.extract_data(api_data)
                if page_data:
                    all_data.extend(page_data)
                    successful_pages += 1
                    tqdm.write(f"第 {page} 页: 提取 {len(page_data)} 条记录")
                else:
                    tqdm.write(f"第 {page} 页: 无数据")
            else:
                tqdm.write(f"第 {page} 页: 调用失败")
            
            # 请求间隔，避免被封
            time.sleep(0.5)
        
        # 保存结果
        if all_data:
            timestamp = time.strftime("%Y%m%d_%H%M%S")
            filename = f"kylin_api_export_{timestamp}.xlsx"
            
            if self.save_to_excel(all_data, filename):
                # 生成统计报告
                self.generate_report(all_data, successful_pages, total_pages, filename)
            else:
                print("✗ 数据保存失败")
        else:
            print("✗ 未获取到任何有效数据")
        
        # 如果有失败的页面，显示失败列表
        if self.failed_pages:
            print(f"\n失败的页面: {self.failed_pages}")
    
    def generate_report(self, data: List[Dict[str, Any]], success_pages: int, total_pages: int, filename: str):
        """
        生成执行报告
        """
        print("\n" + "=" * 50)
        print("执行报告")
        print("=" * 50)
        print(f"总页数: {total_pages}")
        print(f"成功页数: {success_pages}")
        print(f"失败页数: {len(self.failed_pages)}")
        print(f"总记录数: {len(data)}")
        print(f"输出文件: {filename}")
        
        if data:
            # 显示数据结构信息
            sample = data[0]
            print(f"\n数据结构:")
            for key, value in sample.items():
                value_type = type(value).__name__
                value_preview = str(value)[:50] + "..." if len(str(value)) > 50 else str(value)
                print(f"  {key}: {value_type} = {value_preview}")
            
            # 统计各字段的非空数量
            print(f"\n字段统计:")
            df_temp = pd.DataFrame(data)
            for column in df_temp.columns:
                non_null_count = df_temp[column].notna().sum()
                print(f"  {column}: {non_null_count}/{len(data)} 非空")

def main():
    """
    主函数
    """
    print("Kylin API数据导出工具")
    print("版本: 2.0 (增强版)")
    print("=" * 50)
    
    # 用户确认
    total_pages = 200
    input(f"即将调用 {total_pages} 页数据，按Enter键开始...")
    
    # 创建调用器实例
    caller = EnhancedApiCaller()
    
    # 执行调用
    start_time = time.time()
    caller.run_enhanced(total_pages=total_pages, start_page=1)
    end_time = time.time()
    
    print(f"\n总执行时间: {end_time - start_time:.2f} 秒")
    print("程序执行完成！")

if __name__ == "__main__":
    main()